"""
Case Type   : 操作系统类-文件系统故障
Case Name   : gs_basebackup备份/copy恢复时，注入故障系统文件句柄耗尽
Create At   : 2021/8/3
Owner       : li-xin12345
Description :
    1、注入文件系统系统文件句柄耗尽故障
    2、启线程执行gs_basebackup备份
    3、清理故障，再次执行gs_basebackup备份
    4、注入文件系统系统文件句柄耗尽故障
    5、启线程执行copy恢复
    6、清理故障，再次执行copy恢复
    7、运行TPCC
    8、检查数据一致性
    9、恢复环境
Expect      :
    1、故障注入成功
    2、gs_basebackup备份报错“Too many open files”
    3、清理故障成功，gs_basebackup备份成功
    4、故障注入成功
    5、copy恢复报错“Too many open files”
    6、清理故障成功，copy恢复成功
    7、运行TPCC成功
    8、数据一致
    9、恢复环境
History     :
"""
import os
import time
import unittest

from testcase.utils.ComThread import ComThread
from testcase.utils.Common import Common
from testcase.utils.CommonSH import CommonSH
from testcase.utils.CommonSH import RestartDbCluster
from testcase.utils.Constant import Constant
from testcase.utils.Logger import Logger
from yat.test import Node, macro


class ReliabilityFileSystem(unittest.TestCase):
    nodes = ("PrimaryDbUser", "Standby1DbUser", "Standby2DbUser")

    @RestartDbCluster(*nodes)
    def setUp(self):
        self.log = Logger()
        self.primary_node = Node(self.nodes[0])
        self.primary_sh = CommonSH(self.nodes[0])
        self.primary_root_node = Node(node="PrimaryRoot")
        self.constant = Constant()
        self.com = Common()
        self.log.info("setUp-实例化:ok")

        self.log.info("Opengauss_Reliability_OS_File_System_Case039:开始执行")
        self.log.info("创建备份文件夹")
        self.backup_path = os.path.join(os.path.join(
            os.path.dirname(macro.DB_INSTANCE_PATH)),
            "basebackup_dir")
        result = self.com.get_sh_result(self.primary_node,
                                        f"rm -rf {self.backup_path};"
                                        f"mkdir {self.backup_path}")
        self.log.info(result)
        self.dn2 = os.path.join(os.path.abspath(
            os.path.join(macro.DB_INSTANCE_PATH ,
                         os.path.pardir)), "dn2")
        result = self.com.get_sh_result(self.primary_node,
                                        f"rm -rf {self.dn2}; "
                                        f"mkdir {self.dn2}")
        self.log.info(result)

        self.port = self.primary_node.db_port
        self.db_user = self.primary_node.db_user
        self.inject_cmd = f"rfile_deplenish (timeout) values (180)"

    def test_main(self):
        text = "获取hostname"
        result = self.primary_root_node.sh("hostname").result()
        self.log.info(result)
        self.assertNotIn("bash", result, "执行失败" + text)
        self.hostname = result
        try:
            text = "--step1:主节点注入句柄耗尽故障;expect:成功"
            self.log.info(text)
            self.log.info("注入故障")
            session1 = ComThread(self.com.cfe_inject,
                                 args=(self.primary_root_node,
                                       self.inject_cmd))
            session1.setDaemon(True)
            session1.start()
            time.sleep(1)

            self.log.info("故障注入结果")
            session1.join(30)
            result = session1.get_result()
            self.log.info(result)
            self.assertIn("successful execution", result, "执行失败" + text)

            text = "--step2：执行gs_basebackup备份"
            self.log.info(text)
            self.log.info("启线程创建指定实例的备份")
            session2 = ComThread(self.com.get_sh_result,
                                 args=(self.primary_node,
                                       f"source {macro.DB_ENV_PATH};"
                                       f"gs_basebackup -D {self.backup_path}"
                                       f" -h 127.0.0.1 "
                                       f"-p {self.port}"))
            session2.setDaemon(True)
            session2.start()

            self.log.info("gs_basebackup备份结果")
            session2.join(30)
            result = session2.get_result()

            self.log.info('------------------------')
            self.log.info(result)
            self.log.info('***************************')
            self.log.info(self.constant.TOO_MANYY_FILE_MSG)

            self.assertNotIn("successfully", result, "执行失败" + text)
            self.assertIn(self.constant.TOO_MANYY_FILE_MSG, result,
                          "执行失败" + text)
        finally:
            text = "--step3:清除故障;expect:成功"
            self.log.info(text)
            time.sleep(180)
            self.log.info("sleep end")

            result = self.com.cfe_query(self.primary_root_node,
                                        "rfile_deplenish")
            self.log.info(result)
            self.assertNotIn(self.constant.TOO_MANYY_FILE_MSG, result,
                             "执行失败" + text)

            self.log.info("清理备份路径")
            result = self.com.get_sh_result(self.primary_node,
                                            f"rm -rf {self.backup_path};"
                                            f"mkdir {self.backup_path}")
            self.log.info(result)
            self.assertNotIn("bash", result, "执行失败" + text)
            self.assertEqual("", result, "执行失败" + text)

            text = "--step4:再次创建指定实例的备份"
            self.log.info(text)
            res = self.com.get_sh_result(self.primary_node,
                                         f"source {macro.DB_ENV_PATH};"
                                         f"gs_basebackup "
                                         f"-D {self.backup_path}"
                                         f" -h 127.0.0.1 "
                                         f"-p {self.port}")
            self.log.info(res)
            self.assertIn("successfully", res)
            self.assertNotIn(self.constant.TOO_MANYY_FILE_MSG, res)

        self.log.info("gs_basebackup备份测试结束 开始启动测试")

        try:
            result = self.primary_sh.executeGsctl("stop",
                                                  "server stopped",
                                                  get_detail=True)
            self.log.info(result)
            self.assertIn("server stopped", result, "执行失败" + text)

            self.log.info("故障注入结果")
            session1.join(30)
            result = session1.get_result()
            self.log.info(result)
            self.assertIn("successful execution", result, "执行失败" + text)

            text = "--step5:主节点注入句柄耗尽故障;expect:成功"
            self.log.info(text)
            self.log.info("注入故障")
            session1 = ComThread(self.com.cfe_inject,
                                args=(self.primary_root_node,
                                      self.inject_cmd))
            session1.setDaemon(True)
            session1.start()
            time.sleep(1)

            text = "--step5：执行copy 失败"
            self.log.info(text)
            self.log.info("copy")
            sql = f"source {macro.DB_ENV_PATH};" \
                  f"cp -r {self.backup_path}/* {self.dn2}/;" \
                  f"chmod -R 700 {self.dn2};chmod 700 {self.dn2}"
            self.log.info(sql)
            result = self.com.get_sh_result(self.primary_node, sql)
            self.log.info(result)
            self.assertIn(self.constant.TOO_MANYY_FILE_MSG, result,
                          "执行失败" + text)
        finally:
            text = "--step6:清除故障;expect:成功"
            self.log.info(text)
            time.sleep(180)
            self.log.info("sleep end")

            result = self.com.cfe_query(self.primary_root_node,
                                        "rfile_deplenish")
            self.log.info(result)
            self.assertNotIn(self.constant.TOO_MANYY_FILE_MSG, result,
                             "执行失败" + text)

            self.log.info("dn2上启动数据库 失败")
            sql = f"source {macro.DB_ENV_PATH};" \
                  f"gs_ctl start -D {self.dn2}"
            result = self.com.get_sh_result(self.primary_node, sql)
            self.log.info(result)
            self.assertNotIn("server started", result, "执行失败" + text)

            text = "--step6:再次copy 成功"
            self.log.info(text)
            result = self.com.get_sh_result(self.primary_node,
                                            f"rm -rf {self.dn2}; "
                                            f"mkdir {self.dn2}")
            self.log.info(result)
            sql = f"source {macro.DB_ENV_PATH};" \
                  f"cp -r {self.backup_path}/* {self.dn2}/"
            self.log.info(sql)
            result = self.com.get_sh_result(self.primary_node, sql)
            self.log.info(result)
            self.assertNotIn("bash", result, "执行失败" + text)

            self.log.info("设置相关权限")
            cmd = f"chown -R {self.primary_node.ssh_user}:" \
                  f"{self.primary_node.ssh_user} {self.dn2};" \
                  f"chmod -R 700 {self.dn2};chmod 700 {self.dn2}"
            self.log.info(cmd)
            result = self.com.get_sh_result(self.primary_node, cmd)
            self.log.info(result)
            self.assertEqual("", result, "执行失败" + text)

            text = "--step6:再次dn2上启动数据库 成功"
            self.log.info(text)
            sql = f"source {macro.DB_ENV_PATH};" \
                  f"gs_ctl start -D {self.dn2}"
            result = self.com.get_sh_result(self.primary_node, sql)
            self.log.info(result)
            self.assertIn("server started", result, "执行失败" + text)

        sql = f"source {macro.DB_ENV_PATH};" \
              f"gs_ctl stop -D {self.dn2}"
        result = self.com.get_sh_result(self.primary_node, sql)
        self.log.info(result)
        self.assertIn("server stopped", result, "执行失败" + text)

        result = self.primary_sh.executeGsctl("start",
                                              "server started",
                                              get_detail=True)
        self.log.info(result)
        self.assertIn("server started", result, "执行失败" + text)

        db_status = self.primary_sh.getDbClusterStatus("detail")
        self.log.info(db_status)
        self.assertTrue("Degraded" in db_status or "Normal" in db_status)

        text = "--step7:运行TPCC;expect:成功"
        self.log.info(text)
        result = self.com.startTPCC(self.primary_root_node, macro.TPCC_PATH)
        self.log.info(result)
        self.assertIn(self.constant.TPCC_SUCCESS_MSG, result, "执行失败" + text)

        self.log.info("检查主备是否同步")
        time.sleep(10)
        self.log.info("获取节点数量")
        node_num = self.primary_sh.get_node_num()
        res = self.primary_sh.check_location_consistency("primary", node_num)
        self.assertTrue(res, "执行失败" + text)

        text = "--step8:检查数据一致性;expect:成功"
        self.log.info(text)
        nodes = (Node(self.nodes[0]), Node(self.nodes[1]),
                 Node(self.nodes[2]))
        flag = self.com.check_data_sample_by_all(r'\d', *nodes)
        self.assertTrue(flag, "执行失败" + text)
        table_dict = self.com.format_sql_result(
            self.primary_sh.executDbSql(r'\d'))
        table_name = table_dict.get('Name')
        for name in table_name:
            select_sql = f"select count(*) from {name};"
            flag = self.com.check_data_sample_by_all(select_sql, *nodes)
            self.assertTrue(flag, "执行失败" + text)

    def tearDown(self):
        text = "--step9:环境清理;expect:成功"
        self.log.info(text)
        result = self.com.cfe_query(self.primary_root_node,
                                    "rfile_deplenish")
        self.log.info(result)
        if self.constant.TOO_MANYY_FILE_MSG in result:
            time.sleep(300)
            self.log.info("sleep end故障恢复")

        self.log.info("删除相关文件夹")
        result = self.com.get_sh_result(self.primary_node,
                                        f"rm -rf {self.backup_path}")
        self.log.info(result)
        result = Node(self.nodes[0]).sh("date").result()
        self.assertNotIn(self.constant.TOO_MANYY_FILE_MSG, result,
                         "执行失败" + text)
        self.log.info("故障已清理")
        self.log.info("Opengauss_Reliability_OS_File_System_Case039:执行完成")